{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Deep Learning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-05-23T16:52:42.894461Z",
     "start_time": "2017-05-23T16:52:41.091130Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from keras.layers import Input, Dense, Lambda, Layer, LSTM, RepeatVector, Activation\n",
    "from keras.models import Model\n",
    "from keras.layers.core import Dropout\n",
    "from keras import regularizers\n",
    "import keras\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from keras import backend as K\n",
    "from keras import metrics\n",
    "from collections import namedtuple\n",
    "pd.set_option(\"display.max_rows\",35)\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-05-23T16:52:44.059094Z",
     "start_time": "2017-05-23T16:52:42.896263Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "kdd_train_2labels = pd.read_pickle(\"dataset/kdd_train_2labels.pkl\")\n",
    "kdd_test_2labels = pd.read_pickle(\"dataset/kdd_test_2labels.pkl\")\n",
    "\n",
    "#y_train_labels = pd.read_pickle(\"dataset/kdd_train_2labels_y.pkl\")\n",
    "#y_train_labels = pd.read_pickle(\"dataset/kdd_train_2labels.pkl\")\n",
    "#y_test_labels = pd.read_pickle(\"dataset/kdd_test_2labels_y.pkl\")\n",
    "\n",
    "output_columns_2labels = ['is_Attack','is_Normal']\n",
    "\n",
    "from sklearn import model_selection as ms\n",
    "from sklearn import preprocessing as pp\n",
    "\n",
    "x_input = kdd_train_2labels.drop(output_columns_2labels, axis = 1)\n",
    "y_output = kdd_train_2labels.loc[:,output_columns_2labels]\n",
    "\n",
    "ss = pp.StandardScaler()\n",
    "x_input = ss.fit_transform(x_input)\n",
    "\n",
    "#le = pp.LabelEncoder()\n",
    "#y_train = le.fit_transform(y_train_labels).reshape(-1, 1)\n",
    "#y_test = le.transform(y_test_labels).reshape(-1, 1)\n",
    "\n",
    "y_train = kdd_train_2labels.loc[:,output_columns_2labels].values\n",
    "\n",
    "x_train, x_valid, y_train, y_valid = ms.train_test_split(x_input, \n",
    "                              y_train, \n",
    "                              test_size=0.1)\n",
    "#x_valid, x_test, y_valid, y_test = ms.train_test_split(x_valid, y_valid, test_size = 0.4)\n",
    "\n",
    "x_test = kdd_test_2labels.drop(output_columns_2labels, axis = 1)\n",
    "y_test = kdd_test_2labels.loc[:,output_columns_2labels].values\n",
    "\n",
    "x_test = ss.transform(x_test)\n",
    "\n",
    "#x_train = np.hstack((x_train, y_train))\n",
    "#x_valid = np.hstack((x_valid, y_valid))\n",
    "\n",
    "#x_test = np.hstack((x_test, np.random.normal(loc = 0, scale = 0.01, size = y_test.shape)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-05-23T16:52:44.981314Z",
     "start_time": "2017-05-23T16:52:44.060861Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "input_dim = 122\n",
    "intermediate_dim = 10\n",
    "latent_dim = 32\n",
    "batch_size = 1409\n",
    "hidden_layers = 8\n",
    "classes = 2\n",
    "drop_prob = 0.4\n",
    "timesteps = 1\n",
    "\n",
    "class Train:\n",
    "    def build_lstm_model():\n",
    "        Train.x = Input(shape=(timesteps, input_dim))\n",
    "        encoded = LSTM(input_dim)(Train.x)\n",
    "        encoded = Dropout(drop_prob)(encoded)\n",
    "        decoded = RepeatVector(timesteps)(encoded)\n",
    "        Train.y = LSTM(classes, return_sequences=True)(decoded)\n",
    "        Train.y = Activation('softmax')(Train.y)\n",
    "\n",
    "Train.build_lstm_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-05-23T18:21:30.277260Z",
     "start_time": "2017-05-23T18:13:27.169555Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " \n",
      " Current Layer Attributes - epochs:30 hidden layers:6 features count:1\n",
      "Train on 112720 samples, validate on 22544 samples\n",
      "Epoch 1/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2249 - acc: 0.9799Epoch 00000: val_acc improved from -inf to 0.74215, saving model to dataset/epochs_30_hidden layers_6_features count_1\n",
      "112720/112720 [==============================] - 5s - loss: 0.2248 - acc: 0.9799 - val_loss: 0.5729 - val_acc: 0.7421\n",
      "Epoch 2/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2247 - acc: 0.9802Epoch 00001: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2247 - acc: 0.9802 - val_loss: 0.5729 - val_acc: 0.7421\n",
      "Epoch 3/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2247 - acc: 0.9801Epoch 00002: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2248 - acc: 0.9800 - val_loss: 0.5729 - val_acc: 0.7421\n",
      "Epoch 4/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2248 - acc: 0.9801Epoch 00003: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2248 - acc: 0.9801 - val_loss: 0.5728 - val_acc: 0.7421\n",
      "Epoch 5/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2245 - acc: 0.9803Epoch 00004: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2246 - acc: 0.9802 - val_loss: 0.5728 - val_acc: 0.7421\n",
      "Epoch 6/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2249 - acc: 0.9799Epoch 00005: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2248 - acc: 0.9800 - val_loss: 0.5728 - val_acc: 0.7421\n",
      "Epoch 7/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2249 - acc: 0.9799Epoch 00006: val_acc did not improve\n",
      "112720/112720 [==============================] - 3s - loss: 0.2249 - acc: 0.9799 - val_loss: 0.5728 - val_acc: 0.7421\n",
      "Epoch 8/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2247 - acc: 0.9802Epoch 00007: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2248 - acc: 0.9801 - val_loss: 0.5728 - val_acc: 0.7421\n",
      "Epoch 9/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2247 - acc: 0.9801Epoch 00008: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2247 - acc: 0.9801 - val_loss: 0.5728 - val_acc: 0.7421\n",
      "Epoch 10/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2248 - acc: 0.9801Epoch 00009: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2248 - acc: 0.9801 - val_loss: 0.5728 - val_acc: 0.7421\n",
      "Epoch 11/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2248 - acc: 0.9799Epoch 00010: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2248 - acc: 0.9800 - val_loss: 0.5728 - val_acc: 0.7421\n",
      "Epoch 12/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2249 - acc: 0.9800Epoch 00011: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2248 - acc: 0.9801 - val_loss: 0.5728 - val_acc: 0.7421\n",
      "Epoch 13/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2249 - acc: 0.9800Epoch 00012: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2248 - acc: 0.9800 - val_loss: 0.5728 - val_acc: 0.7421\n",
      "Epoch 14/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2248 - acc: 0.9800Epoch 00013: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2247 - acc: 0.9801 - val_loss: 0.5728 - val_acc: 0.7421\n",
      "Epoch 15/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2249 - acc: 0.9800Epoch 00014: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2248 - acc: 0.9801 - val_loss: 0.5728 - val_acc: 0.7421\n",
      "Epoch 16/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2248 - acc: 0.9799Epoch 00015: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2247 - acc: 0.9799 - val_loss: 0.5728 - val_acc: 0.7421\n",
      "Epoch 17/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2246 - acc: 0.9800Epoch 00016: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2247 - acc: 0.9800 - val_loss: 0.5728 - val_acc: 0.7421\n",
      "Epoch 18/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2247 - acc: 0.9801Epoch 00017: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2248 - acc: 0.9800 - val_loss: 0.5728 - val_acc: 0.7421\n",
      "Epoch 19/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2248 - acc: 0.9800Epoch 00018: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2248 - acc: 0.9800 - val_loss: 0.5728 - val_acc: 0.7421\n",
      "Epoch 20/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2246 - acc: 0.9800Epoch 00019: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2246 - acc: 0.9800 - val_loss: 0.5728 - val_acc: 0.7421\n",
      "Epoch 21/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2248 - acc: 0.9800Epoch 00020: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2248 - acc: 0.9800 - val_loss: 0.5728 - val_acc: 0.7421\n",
      "Epoch 22/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2247 - acc: 0.9801Epoch 00021: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2247 - acc: 0.9800 - val_loss: 0.5728 - val_acc: 0.7421\n",
      "Epoch 23/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2248 - acc: 0.9801Epoch 00022: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2247 - acc: 0.9801 - val_loss: 0.5728 - val_acc: 0.7421\n",
      "Epoch 24/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2248 - acc: 0.9799Epoch 00023: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2247 - acc: 0.9799 - val_loss: 0.5728 - val_acc: 0.7421\n",
      "Epoch 25/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2249 - acc: 0.9799Epoch 00024: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2248 - acc: 0.9799 - val_loss: 0.5728 - val_acc: 0.7421\n",
      "Epoch 26/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2247 - acc: 0.9800Epoch 00025: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2248 - acc: 0.9800 - val_loss: 0.5728 - val_acc: 0.7421\n",
      "Epoch 27/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2248 - acc: 0.9800Epoch 00026: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2248 - acc: 0.9800 - val_loss: 0.5728 - val_acc: 0.7421\n",
      "Epoch 28/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2248 - acc: 0.9799Epoch 00027: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2248 - acc: 0.9799 - val_loss: 0.5728 - val_acc: 0.7421\n",
      "Epoch 29/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2248 - acc: 0.9800Epoch 00028: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2248 - acc: 0.9800 - val_loss: 0.5728 - val_acc: 0.7421\n",
      "Epoch 30/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2248 - acc: 0.9801Epoch 00029: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2247 - acc: 0.9802 - val_loss: 0.5728 - val_acc: 0.7421\n",
      "18317/22544 [=======================>......] - ETA: 0s\n",
      " Train Acc: 0.9819907695055008, Test Acc: 0.7421043328940868\n",
      " \n",
      " Current Layer Attributes - epochs:30 hidden layers:8 features count:1\n",
      "Train on 112720 samples, validate on 22544 samples\n",
      "Epoch 1/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2247 - acc: 0.9800Epoch 00000: val_acc improved from -inf to 0.74179, saving model to dataset/epochs_30_hidden layers_8_features count_1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "112720/112720 [==============================] - 5s - loss: 0.2248 - acc: 0.9800 - val_loss: 0.5730 - val_acc: 0.7418\n",
      "Epoch 2/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2247 - acc: 0.9801Epoch 00001: val_acc improved from 0.74179 to 0.74184, saving model to dataset/epochs_30_hidden layers_8_features count_1\n",
      "112720/112720 [==============================] - 3s - loss: 0.2248 - acc: 0.9800 - val_loss: 0.5730 - val_acc: 0.7418\n",
      "Epoch 3/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2248 - acc: 0.9800Epoch 00002: val_acc improved from 0.74184 to 0.74188, saving model to dataset/epochs_30_hidden layers_8_features count_1\n",
      "112720/112720 [==============================] - 3s - loss: 0.2248 - acc: 0.9801 - val_loss: 0.5729 - val_acc: 0.7419\n",
      "Epoch 4/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2247 - acc: 0.9802Epoch 00003: val_acc did not improve\n",
      "112720/112720 [==============================] - 3s - loss: 0.2248 - acc: 0.9801 - val_loss: 0.5729 - val_acc: 0.7419\n",
      "Epoch 5/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2248 - acc: 0.9800Epoch 00004: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2248 - acc: 0.9800 - val_loss: 0.5729 - val_acc: 0.7418\n",
      "Epoch 6/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2248 - acc: 0.9800Epoch 00005: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2247 - acc: 0.9800 - val_loss: 0.5729 - val_acc: 0.7418\n",
      "Epoch 7/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2248 - acc: 0.9800Epoch 00006: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2247 - acc: 0.9801 - val_loss: 0.5729 - val_acc: 0.7418\n",
      "Epoch 8/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2247 - acc: 0.9801Epoch 00007: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2247 - acc: 0.9801 - val_loss: 0.5729 - val_acc: 0.7418\n",
      "Epoch 9/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2250 - acc: 0.9799Epoch 00008: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2249 - acc: 0.9800 - val_loss: 0.5729 - val_acc: 0.7418\n",
      "Epoch 10/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2249 - acc: 0.9800Epoch 00009: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2248 - acc: 0.9801 - val_loss: 0.5729 - val_acc: 0.7419\n",
      "Epoch 11/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2249 - acc: 0.9800Epoch 00010: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2248 - acc: 0.9800 - val_loss: 0.5729 - val_acc: 0.7418\n",
      "Epoch 12/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2248 - acc: 0.9799Epoch 00011: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2248 - acc: 0.9799 - val_loss: 0.5729 - val_acc: 0.7418\n",
      "Epoch 13/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2247 - acc: 0.9799Epoch 00012: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2247 - acc: 0.9799 - val_loss: 0.5729 - val_acc: 0.7418\n",
      "Epoch 14/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2248 - acc: 0.9800Epoch 00013: val_acc did not improve\n",
      "112720/112720 [==============================] - 3s - loss: 0.2248 - acc: 0.9800 - val_loss: 0.5729 - val_acc: 0.7418\n",
      "Epoch 15/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2247 - acc: 0.9800Epoch 00014: val_acc did not improve\n",
      "112720/112720 [==============================] - 3s - loss: 0.2248 - acc: 0.9799 - val_loss: 0.5729 - val_acc: 0.7418\n",
      "Epoch 16/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2246 - acc: 0.9800Epoch 00015: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2247 - acc: 0.9801 - val_loss: 0.5729 - val_acc: 0.7418\n",
      "Epoch 17/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2248 - acc: 0.9799Epoch 00016: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2248 - acc: 0.9799 - val_loss: 0.5729 - val_acc: 0.7418\n",
      "Epoch 18/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2248 - acc: 0.9798Epoch 00017: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2247 - acc: 0.9799 - val_loss: 0.5729 - val_acc: 0.7418\n",
      "Epoch 19/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2247 - acc: 0.9801Epoch 00018: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2247 - acc: 0.9800 - val_loss: 0.5729 - val_acc: 0.7418\n",
      "Epoch 20/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2247 - acc: 0.9801Epoch 00019: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2246 - acc: 0.9802 - val_loss: 0.5729 - val_acc: 0.7418\n",
      "Epoch 21/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2246 - acc: 0.9801Epoch 00020: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2247 - acc: 0.9801 - val_loss: 0.5729 - val_acc: 0.7419\n",
      "Epoch 22/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2247 - acc: 0.9801Epoch 00021: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2246 - acc: 0.9801 - val_loss: 0.5729 - val_acc: 0.7419\n",
      "Epoch 23/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2248 - acc: 0.9800Epoch 00022: val_acc did not improve\n",
      "112720/112720 [==============================] - 3s - loss: 0.2248 - acc: 0.9800 - val_loss: 0.5729 - val_acc: 0.7419\n",
      "Epoch 24/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2247 - acc: 0.9801Epoch 00023: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2247 - acc: 0.9801 - val_loss: 0.5729 - val_acc: 0.7419\n",
      "Epoch 25/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2246 - acc: 0.9801Epoch 00024: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2247 - acc: 0.9801 - val_loss: 0.5729 - val_acc: 0.7419\n",
      "Epoch 26/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2248 - acc: 0.9799Epoch 00025: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2248 - acc: 0.9800 - val_loss: 0.5729 - val_acc: 0.7419\n",
      "Epoch 27/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2246 - acc: 0.9802Epoch 00026: val_acc did not improve\n",
      "112720/112720 [==============================] - 3s - loss: 0.2246 - acc: 0.9801 - val_loss: 0.5729 - val_acc: 0.7419\n",
      "Epoch 28/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2248 - acc: 0.9801Epoch 00027: val_acc did not improve\n",
      "112720/112720 [==============================] - 3s - loss: 0.2248 - acc: 0.9801 - val_loss: 0.5729 - val_acc: 0.7418\n",
      "Epoch 29/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2246 - acc: 0.9801Epoch 00028: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2247 - acc: 0.9801 - val_loss: 0.5729 - val_acc: 0.7418\n",
      "Epoch 30/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2248 - acc: 0.9799Epoch 00029: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2248 - acc: 0.9800 - val_loss: 0.5729 - val_acc: 0.7419\n",
      "18317/22544 [=======================>......] - ETA: 0s\n",
      " Train Acc: 0.9819907695055008, Test Acc: 0.7418825440108776\n",
      " \n",
      " Current Layer Attributes - epochs:30 hidden layers:16 features count:1\n",
      "Train on 112720 samples, validate on 22544 samples\n",
      "Epoch 1/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2249 - acc: 0.9799Epoch 00000: val_acc improved from -inf to 0.74210, saving model to dataset/epochs_30_hidden layers_16_features count_1\n",
      "112720/112720 [==============================] - 5s - loss: 0.2248 - acc: 0.9800 - val_loss: 0.5725 - val_acc: 0.7421\n",
      "Epoch 2/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2248 - acc: 0.9801Epoch 00001: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2248 - acc: 0.9800 - val_loss: 0.5725 - val_acc: 0.7421\n",
      "Epoch 3/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2248 - acc: 0.9800Epoch 00002: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2247 - acc: 0.9801 - val_loss: 0.5725 - val_acc: 0.7421\n",
      "Epoch 4/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2247 - acc: 0.9801Epoch 00003: val_acc did not improve\n",
      "112720/112720 [==============================] - 3s - loss: 0.2247 - acc: 0.9801 - val_loss: 0.5725 - val_acc: 0.7421\n",
      "Epoch 5/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2245 - acc: 0.9802Epoch 00004: val_acc did not improve\n",
      "112720/112720 [==============================] - 3s - loss: 0.2246 - acc: 0.9802 - val_loss: 0.5725 - val_acc: 0.7421\n",
      "Epoch 6/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2247 - acc: 0.9800Epoch 00005: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2247 - acc: 0.9800 - val_loss: 0.5725 - val_acc: 0.7421\n",
      "Epoch 7/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2247 - acc: 0.9800Epoch 00006: val_acc did not improve\n",
      "112720/112720 [==============================] - 3s - loss: 0.2247 - acc: 0.9800 - val_loss: 0.5725 - val_acc: 0.7421\n",
      "Epoch 8/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2247 - acc: 0.9800Epoch 00007: val_acc did not improve\n",
      "112720/112720 [==============================] - 3s - loss: 0.2248 - acc: 0.9799 - val_loss: 0.5725 - val_acc: 0.7421\n",
      "Epoch 9/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2247 - acc: 0.9801Epoch 00008: val_acc did not improve\n",
      "112720/112720 [==============================] - 3s - loss: 0.2247 - acc: 0.9801 - val_loss: 0.5725 - val_acc: 0.7421\n",
      "Epoch 10/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2248 - acc: 0.9799Epoch 00009: val_acc did not improve\n",
      "112720/112720 [==============================] - 3s - loss: 0.2247 - acc: 0.9800 - val_loss: 0.5725 - val_acc: 0.7421\n",
      "Epoch 11/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2248 - acc: 0.9801Epoch 00010: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2247 - acc: 0.9800 - val_loss: 0.5725 - val_acc: 0.7421\n",
      "Epoch 12/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2248 - acc: 0.9800Epoch 00011: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2247 - acc: 0.9800 - val_loss: 0.5725 - val_acc: 0.7421\n",
      "Epoch 13/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2247 - acc: 0.9801Epoch 00012: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2247 - acc: 0.9801 - val_loss: 0.5725 - val_acc: 0.7421\n",
      "Epoch 14/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2247 - acc: 0.9801Epoch 00013: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2247 - acc: 0.9801 - val_loss: 0.5725 - val_acc: 0.7421\n",
      "Epoch 15/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2246 - acc: 0.9802Epoch 00014: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2247 - acc: 0.9801 - val_loss: 0.5725 - val_acc: 0.7421\n",
      "Epoch 16/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2248 - acc: 0.9800Epoch 00015: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2248 - acc: 0.9799 - val_loss: 0.5725 - val_acc: 0.7421\n",
      "Epoch 17/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2247 - acc: 0.9800Epoch 00016: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2247 - acc: 0.9800 - val_loss: 0.5725 - val_acc: 0.7421\n",
      "Epoch 18/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2247 - acc: 0.9800Epoch 00017: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2246 - acc: 0.9801 - val_loss: 0.5725 - val_acc: 0.7421\n",
      "Epoch 19/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2246 - acc: 0.9801Epoch 00018: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2246 - acc: 0.9801 - val_loss: 0.5725 - val_acc: 0.7421\n",
      "Epoch 20/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2248 - acc: 0.9800Epoch 00019: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2247 - acc: 0.9800 - val_loss: 0.5725 - val_acc: 0.7421\n",
      "Epoch 21/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2248 - acc: 0.9799Epoch 00020: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2247 - acc: 0.9799 - val_loss: 0.5725 - val_acc: 0.7421\n",
      "Epoch 22/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2247 - acc: 0.9800Epoch 00021: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2247 - acc: 0.9800 - val_loss: 0.5725 - val_acc: 0.7421\n",
      "Epoch 23/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2248 - acc: 0.9801Epoch 00022: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2247 - acc: 0.9801 - val_loss: 0.5725 - val_acc: 0.7421\n",
      "Epoch 24/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2247 - acc: 0.9800Epoch 00023: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2247 - acc: 0.9800 - val_loss: 0.5725 - val_acc: 0.7421\n",
      "Epoch 25/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2247 - acc: 0.9801Epoch 00024: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2247 - acc: 0.9801 - val_loss: 0.5725 - val_acc: 0.7421\n",
      "Epoch 26/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2247 - acc: 0.9800Epoch 00025: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2247 - acc: 0.9799 - val_loss: 0.5725 - val_acc: 0.7421\n",
      "Epoch 27/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2247 - acc: 0.9800Epoch 00026: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2247 - acc: 0.9800 - val_loss: 0.5725 - val_acc: 0.7421\n",
      "Epoch 28/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2247 - acc: 0.9801Epoch 00027: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2247 - acc: 0.9801 - val_loss: 0.5725 - val_acc: 0.7421\n",
      "Epoch 29/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2246 - acc: 0.9802Epoch 00028: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2247 - acc: 0.9800 - val_loss: 0.5725 - val_acc: 0.7421\n",
      "Epoch 30/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2248 - acc: 0.9799Epoch 00029: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2247 - acc: 0.9800 - val_loss: 0.5725 - val_acc: 0.7421\n",
      "18317/22544 [=======================>......] - ETA: 0s\n",
      " Train Acc: 0.9819907695055008, Test Acc: 0.7421043291687965\n",
      " \n",
      " Current Layer Attributes - epochs:30 hidden layers:32 features count:1\n",
      "Train on 112720 samples, validate on 22544 samples\n",
      "Epoch 1/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2246 - acc: 0.9801Epoch 00000: val_acc improved from -inf to 0.74215, saving model to dataset/epochs_30_hidden layers_32_features count_1\n",
      "112720/112720 [==============================] - 5s - loss: 0.2247 - acc: 0.9800 - val_loss: 0.5725 - val_acc: 0.7421\n",
      "Epoch 2/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2246 - acc: 0.9802Epoch 00001: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2246 - acc: 0.9802 - val_loss: 0.5725 - val_acc: 0.7421\n",
      "Epoch 3/30\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2247 - acc: 0.9800Epoch 00002: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2247 - acc: 0.9800 - val_loss: 0.5725 - val_acc: 0.7421\n",
      "Epoch 4/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2247 - acc: 0.9800Epoch 00003: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2247 - acc: 0.9800 - val_loss: 0.5725 - val_acc: 0.7421\n",
      "Epoch 5/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2247 - acc: 0.9800Epoch 00004: val_acc did not improve\n",
      "112720/112720 [==============================] - 3s - loss: 0.2247 - acc: 0.9801 - val_loss: 0.5725 - val_acc: 0.7421\n",
      "Epoch 6/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2247 - acc: 0.9800Epoch 00005: val_acc did not improve\n",
      "112720/112720 [==============================] - 3s - loss: 0.2246 - acc: 0.9802 - val_loss: 0.5725 - val_acc: 0.7421\n",
      "Epoch 7/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2246 - acc: 0.9801Epoch 00006: val_acc did not improve\n",
      "112720/112720 [==============================] - 3s - loss: 0.2247 - acc: 0.9800 - val_loss: 0.5725 - val_acc: 0.7421\n",
      "Epoch 8/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2248 - acc: 0.9800Epoch 00007: val_acc did not improve\n",
      "112720/112720 [==============================] - 3s - loss: 0.2248 - acc: 0.9800 - val_loss: 0.5725 - val_acc: 0.7421\n",
      "Epoch 9/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2246 - acc: 0.9801Epoch 00008: val_acc did not improve\n",
      "112720/112720 [==============================] - 3s - loss: 0.2246 - acc: 0.9801 - val_loss: 0.5725 - val_acc: 0.7421\n",
      "Epoch 10/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2247 - acc: 0.9802Epoch 00009: val_acc did not improve\n",
      "112720/112720 [==============================] - 3s - loss: 0.2247 - acc: 0.9802 - val_loss: 0.5725 - val_acc: 0.7421\n",
      "Epoch 11/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2247 - acc: 0.9801Epoch 00010: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2247 - acc: 0.9801 - val_loss: 0.5725 - val_acc: 0.7421\n",
      "Epoch 12/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2247 - acc: 0.9801Epoch 00011: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2246 - acc: 0.9802 - val_loss: 0.5725 - val_acc: 0.7421\n",
      "Epoch 13/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2245 - acc: 0.9801Epoch 00012: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2246 - acc: 0.9801 - val_loss: 0.5725 - val_acc: 0.7421\n",
      "Epoch 14/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2247 - acc: 0.9801Epoch 00013: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2246 - acc: 0.9801 - val_loss: 0.5725 - val_acc: 0.7421\n",
      "Epoch 15/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2247 - acc: 0.9801Epoch 00014: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2247 - acc: 0.9801 - val_loss: 0.5725 - val_acc: 0.7421\n",
      "Epoch 16/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2247 - acc: 0.9800Epoch 00015: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2247 - acc: 0.9801 - val_loss: 0.5725 - val_acc: 0.7421\n",
      "Epoch 17/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2246 - acc: 0.9801Epoch 00016: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2246 - acc: 0.9801 - val_loss: 0.5725 - val_acc: 0.7421\n",
      "Epoch 18/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2247 - acc: 0.9801Epoch 00017: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2247 - acc: 0.9801 - val_loss: 0.5725 - val_acc: 0.7421\n",
      "Epoch 19/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2247 - acc: 0.9800Epoch 00018: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2247 - acc: 0.9800 - val_loss: 0.5725 - val_acc: 0.7421\n",
      "Epoch 20/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2245 - acc: 0.9802Epoch 00019: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2246 - acc: 0.9802 - val_loss: 0.5725 - val_acc: 0.7421\n",
      "Epoch 21/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2246 - acc: 0.9801Epoch 00020: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2246 - acc: 0.9800 - val_loss: 0.5725 - val_acc: 0.7421\n",
      "Epoch 22/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2248 - acc: 0.9798Epoch 00021: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2248 - acc: 0.9799 - val_loss: 0.5725 - val_acc: 0.7421\n",
      "Epoch 23/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2246 - acc: 0.9801Epoch 00022: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2247 - acc: 0.9801 - val_loss: 0.5725 - val_acc: 0.7421\n",
      "Epoch 24/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2245 - acc: 0.9802Epoch 00023: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2246 - acc: 0.9801 - val_loss: 0.5725 - val_acc: 0.7421\n",
      "Epoch 25/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2246 - acc: 0.9802Epoch 00024: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2247 - acc: 0.9802 - val_loss: 0.5725 - val_acc: 0.7421\n",
      "Epoch 26/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2247 - acc: 0.9801Epoch 00025: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2246 - acc: 0.9801 - val_loss: 0.5725 - val_acc: 0.7421\n",
      "Epoch 27/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2247 - acc: 0.9800Epoch 00026: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2247 - acc: 0.9799 - val_loss: 0.5725 - val_acc: 0.7421\n",
      "Epoch 28/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2246 - acc: 0.9801Epoch 00027: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2246 - acc: 0.9802 - val_loss: 0.5725 - val_acc: 0.7421\n",
      "Epoch 29/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2246 - acc: 0.9800Epoch 00028: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2246 - acc: 0.9800 - val_loss: 0.5725 - val_acc: 0.7421\n",
      "Epoch 30/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2247 - acc: 0.9802Epoch 00029: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2247 - acc: 0.9802 - val_loss: 0.5725 - val_acc: 0.7421\n",
      "18317/22544 [=======================>......] - ETA: 0s\n",
      " Train Acc: 0.9819907695055008, Test Acc: 0.7421486899256706\n",
      " \n",
      " Current Layer Attributes - epochs:30 hidden layers:64 features count:1\n",
      "Train on 112720 samples, validate on 22544 samples\n",
      "Epoch 1/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2246 - acc: 0.9801Epoch 00000: val_acc improved from -inf to 0.74219, saving model to dataset/epochs_30_hidden layers_64_features count_1\n",
      "112720/112720 [==============================] - 5s - loss: 0.2247 - acc: 0.9801 - val_loss: 0.5725 - val_acc: 0.7422\n",
      "Epoch 2/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2247 - acc: 0.9801Epoch 00001: val_acc did not improve\n",
      "112720/112720 [==============================] - 3s - loss: 0.2246 - acc: 0.9801 - val_loss: 0.5725 - val_acc: 0.7422\n",
      "Epoch 3/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2245 - acc: 0.9802Epoch 00002: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2245 - acc: 0.9802 - val_loss: 0.5725 - val_acc: 0.7422\n",
      "Epoch 4/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2248 - acc: 0.9799Epoch 00003: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2247 - acc: 0.9800 - val_loss: 0.5724 - val_acc: 0.7422\n",
      "Epoch 5/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2247 - acc: 0.9800Epoch 00004: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2247 - acc: 0.9800 - val_loss: 0.5724 - val_acc: 0.7422\n",
      "Epoch 6/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2246 - acc: 0.9801Epoch 00005: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2246 - acc: 0.9800 - val_loss: 0.5724 - val_acc: 0.7422\n",
      "Epoch 7/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2245 - acc: 0.9802Epoch 00006: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2246 - acc: 0.9800 - val_loss: 0.5724 - val_acc: 0.7422\n",
      "Epoch 8/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2246 - acc: 0.9802Epoch 00007: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2245 - acc: 0.9802 - val_loss: 0.5724 - val_acc: 0.7422\n",
      "Epoch 9/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2246 - acc: 0.9801Epoch 00008: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2246 - acc: 0.9801 - val_loss: 0.5724 - val_acc: 0.7422\n",
      "Epoch 10/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2246 - acc: 0.9802Epoch 00009: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2245 - acc: 0.9803 - val_loss: 0.5724 - val_acc: 0.7422\n",
      "Epoch 11/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2247 - acc: 0.9801Epoch 00010: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2247 - acc: 0.9801 - val_loss: 0.5724 - val_acc: 0.7422\n",
      "Epoch 12/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2246 - acc: 0.9801Epoch 00011: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2246 - acc: 0.9801 - val_loss: 0.5724 - val_acc: 0.7422\n",
      "Epoch 13/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2248 - acc: 0.9800Epoch 00012: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2246 - acc: 0.9801 - val_loss: 0.5724 - val_acc: 0.7422\n",
      "Epoch 14/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2246 - acc: 0.9800Epoch 00013: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2246 - acc: 0.9800 - val_loss: 0.5724 - val_acc: 0.7422\n",
      "Epoch 15/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2245 - acc: 0.9801Epoch 00014: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2246 - acc: 0.9801 - val_loss: 0.5724 - val_acc: 0.7422\n",
      "Epoch 16/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2247 - acc: 0.9802Epoch 00015: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2246 - acc: 0.9802 - val_loss: 0.5724 - val_acc: 0.7422\n",
      "Epoch 17/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2248 - acc: 0.9800Epoch 00016: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2247 - acc: 0.9801 - val_loss: 0.5724 - val_acc: 0.7422\n",
      "Epoch 18/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2246 - acc: 0.9800Epoch 00017: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2247 - acc: 0.9801 - val_loss: 0.5724 - val_acc: 0.7422\n",
      "Epoch 19/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2246 - acc: 0.9802Epoch 00018: val_acc improved from 0.74219 to 0.74224, saving model to dataset/epochs_30_hidden layers_64_features count_1\n",
      "112720/112720 [==============================] - 2s - loss: 0.2246 - acc: 0.9801 - val_loss: 0.5724 - val_acc: 0.7422\n",
      "Epoch 20/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2247 - acc: 0.9801Epoch 00019: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2247 - acc: 0.9801 - val_loss: 0.5724 - val_acc: 0.7422\n",
      "Epoch 21/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2245 - acc: 0.9801Epoch 00020: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2245 - acc: 0.9802 - val_loss: 0.5724 - val_acc: 0.7422\n",
      "Epoch 22/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2246 - acc: 0.9799Epoch 00021: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2246 - acc: 0.9799 - val_loss: 0.5724 - val_acc: 0.7422\n",
      "Epoch 23/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2247 - acc: 0.9800Epoch 00022: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2247 - acc: 0.9801 - val_loss: 0.5724 - val_acc: 0.7422\n",
      "Epoch 24/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2247 - acc: 0.9800Epoch 00023: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2246 - acc: 0.9800 - val_loss: 0.5724 - val_acc: 0.7422\n",
      "Epoch 25/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2246 - acc: 0.9801Epoch 00024: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2246 - acc: 0.9801 - val_loss: 0.5724 - val_acc: 0.7422\n",
      "Epoch 26/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2245 - acc: 0.9803Epoch 00025: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2246 - acc: 0.9802 - val_loss: 0.5724 - val_acc: 0.7422\n",
      "Epoch 27/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2248 - acc: 0.9800Epoch 00026: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2246 - acc: 0.9801 - val_loss: 0.5724 - val_acc: 0.7422\n",
      "Epoch 28/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2247 - acc: 0.9800Epoch 00027: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2247 - acc: 0.9801 - val_loss: 0.5724 - val_acc: 0.7422\n",
      "Epoch 29/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2246 - acc: 0.9802Epoch 00028: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2245 - acc: 0.9802 - val_loss: 0.5724 - val_acc: 0.7422\n",
      "Epoch 30/30\n",
      "111311/112720 [============================>.] - ETA: 0s - loss: 0.2247 - acc: 0.9799Epoch 00029: val_acc did not improve\n",
      "112720/112720 [==============================] - 2s - loss: 0.2246 - acc: 0.9800 - val_loss: 0.5724 - val_acc: 0.7422\n",
      "18317/22544 [=======================>......] - ETA: 0s\n",
      " Train Acc: 0.9819907695055008, Test Acc: 0.7422374039888382\n"
     ]
    }
   ],
   "source": [
    "import itertools\n",
    "#features_arr = [4, 16, 32, 256, 1024]\n",
    "#hidden_layers_arr = [2, 6, 10, 100]\n",
    "\n",
    "#features_arr = [4, 16, 32]\n",
    "#hidden_layers_arr = [2, 6, 10]\n",
    "\n",
    "features_arr = [1] # [4, 16, 32]\n",
    "hidden_layers_arr = [6, 8, 16, 32, 64]\n",
    "\n",
    "epoch_arr = [30]\n",
    "\n",
    "score = namedtuple(\"score\", ['epoch', 'no_of_features','hidden_layers','train_score', 'test_score'])\n",
    "scores = []\n",
    "predictions = {}\n",
    "\n",
    "for e, h, f in itertools.product(epoch_arr, hidden_layers_arr, features_arr):\n",
    "    \n",
    "    print(\" \\n Current Layer Attributes - epochs:{} hidden layers:{} features count:{}\".format(e,h,f))\n",
    "    latent_dim = f\n",
    "    epochs = e\n",
    "    hidden_layers = h\n",
    "    \n",
    "    train_size = x_train.shape[0] - x_train.shape[0]%batch_size\n",
    "    valid_size = x_valid.shape[0] - x_valid.shape[0]%batch_size\n",
    "\n",
    "    \n",
    "    #optimizer = keras.optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-04, decay=0.1)\n",
    "    optimizer = keras.optimizers.RMSprop(lr=0.0001, decay=0.1)\n",
    "    \n",
    "    seq2seq_model = Model(Train.x, Train.y)\n",
    "    seq2seq_model.compile(optimizer = optimizer, \n",
    "                      loss = keras.losses.categorical_crossentropy, \n",
    "                      metrics = ['accuracy'])\n",
    "    \n",
    "    ckp = keras.callbacks.ModelCheckpoint(\"dataset/epochs_{}_hidden layers_{}_features count_{}\".format(e,h,f), \n",
    "                                          monitor='val_acc', verbose=1, \n",
    "                                          save_best_only=True, save_weights_only=False, \n",
    "                                          mode='auto', period=1)\n",
    "\n",
    "    seq2seq_model.fit(x = x_train[:train_size,np.newaxis,:], y = y_train[:train_size,np.newaxis,:],\n",
    "                 shuffle=True, epochs=epochs, \n",
    "                  batch_size = batch_size, \n",
    "                  validation_data = (x_test[:,np.newaxis,:], y_test[:,np.newaxis,:]),\n",
    "                  verbose = 1, callbacks=[ckp])\n",
    "\n",
    "    score_train = seq2seq_model.evaluate(x_valid[:valid_size,np.newaxis,:], y = y_valid[:valid_size,np.newaxis,:],\n",
    "                               batch_size = batch_size,\n",
    "                               verbose = 1)\n",
    "    \n",
    "    score_test = seq2seq_model.evaluate(x_test[:,np.newaxis,:], y = y_test[:,np.newaxis,:],\n",
    "                           batch_size = batch_size,\n",
    "                           verbose = 1)\n",
    "    \n",
    "    y_test_pred = seq2seq_model.predict(x_test[:,np.newaxis,:], batch_size=batch_size)\n",
    "    y_test_pred = np.squeeze(y_test_pred)\n",
    "\n",
    "    y_pred = y_test_pred #np.argmax(y_test_pred[:,-2:], axis = 1)\n",
    "    \n",
    "    curr_pred = pd.DataFrame({\"Attack_prob\":y_pred[:,0], \"Normal_prob\":y_pred[:,1]})\n",
    "    predictions.update({\"{}_{}_{}\".format(e,f,h):curr_pred})\n",
    "    \n",
    "    scores.append(score(e,f,h,score_train[-1], score_test[-1])) #score_test[-1]))\n",
    "    \n",
    "    print(\"\\n Train Acc: {}, Test Acc: {}\".format(score_train[-1], \n",
    "                                                  score_test[-1])  )\n",
    "    \n",
    "scores = pd.DataFrame(scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-05-23T18:21:30.289907Z",
     "start_time": "2017-05-23T18:21:30.278952Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>epoch</th>\n",
       "      <th>no_of_features</th>\n",
       "      <th>hidden_layers</th>\n",
       "      <th>train_score</th>\n",
       "      <th>test_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>30</td>\n",
       "      <td>1</td>\n",
       "      <td>64</td>\n",
       "      <td>0.981991</td>\n",
       "      <td>0.742237</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>30</td>\n",
       "      <td>1</td>\n",
       "      <td>32</td>\n",
       "      <td>0.981991</td>\n",
       "      <td>0.742149</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>30</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>0.981991</td>\n",
       "      <td>0.742104</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>30</td>\n",
       "      <td>1</td>\n",
       "      <td>16</td>\n",
       "      <td>0.981991</td>\n",
       "      <td>0.742104</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>30</td>\n",
       "      <td>1</td>\n",
       "      <td>8</td>\n",
       "      <td>0.981991</td>\n",
       "      <td>0.741883</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   epoch  no_of_features  hidden_layers  train_score  test_score\n",
       "4     30               1             64     0.981991    0.742237\n",
       "3     30               1             32     0.981991    0.742149\n",
       "0     30               1              6     0.981991    0.742104\n",
       "2     30               1             16     0.981991    0.742104\n",
       "1     30               1              8     0.981991    0.741883"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores.sort_values(\"test_score\", ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-05-23T18:21:30.334261Z",
     "start_time": "2017-05-23T18:21:30.291530Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pd.Panel(predictions).to_pickle(\"dataset/keras_lstm_nsl_kdd_predictions.pkl\")\n",
    "scores.to_pickle(\"dataset/keras_lstm_nsl_kdd_scores.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-05-23T18:21:30.343349Z",
     "start_time": "2017-05-23T18:21:30.335669Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<class 'pandas.core.panel.Panel'>\n",
       "Dimensions: 5 (items) x 22544 (major_axis) x 2 (minor_axis)\n",
       "Items axis: 30_1_16 to 30_1_8\n",
       "Major_axis axis: 0 to 22543\n",
       "Minor_axis axis: Attack_prob to Normal_prob"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.Panel(predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "_draft": {
   "nbviewer_url": "https://gist.github.com/33dcb1bcf3ca4a3461c4405a003a7591"
  },
  "anaconda-cloud": {},
  "gist": {
   "data": {
    "description": "Final Hyper parameter tuning",
    "public": false
   },
   "id": "33dcb1bcf3ca4a3461c4405a003a7591"
  },
  "kernelspec": {
   "display_name": "Python [conda env:p3]",
   "language": "python",
   "name": "conda-env-p3-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
